package cn.sheep.violet.tools

import cn.sheep.violet.config.ConfigHandler
import cn.sheep.violet.utils.Jpools
import org.apache.commons.lang.StringUtils
import org.apache.spark.{SparkConf, SparkContext}

/** 将app字典数据送入到redis中
  * author: old sheep
  * QQ: 64341393 
  * Created 2018/10/17
  */
object Appdict2Redis {

    def main(args: Array[String]): Unit = {

        val sparkConf = new SparkConf()
            .setAppName(this.getClass.getSimpleName)
            .setMaster("local[*]")
            .set("", "") //Kryo
        val sc = new SparkContext(sparkConf)

        // 读取字典数据
        sc.textFile(ConfigHandler.appdictFilePath)
                .map(_.split("\t", -1))
                .filter(_.length >= 5)
                .foreachPartition(iter => {
                    val jedis = Jpools.getJedis
                    iter.foreach(data => {
                        val appName = data(1)
                        val appId = data(4)
                        if (StringUtils.isNotEmpty(appId)) jedis.hset("appdict", appId, appName)
                    })
                    jedis.close()
                })
        sc.stop()
    }

}
